package common.main.cnkiJournalPaper;

import common.http.HtmlInfo;
import common.http.SimpleHttp;
import common.system.FileOperation;
import common.utils.DomTree;
import common.utils.MD5Util;
import org.apache.log4j.Logger;
import org.w3c.dom.Document;
import org.w3c.dom.DocumentFragment;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.SAXException;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.text.SimpleDateFormat;
import java.util.*;

public class CnkiJournalPaperMain {
    public static Logger logger = Logger.getLogger(CnkiJournalPaperMain.class);

    public static String urlPrefix = "http://navi.cnki.net";
    public static String SearchStateJson = "{\"StateID\":\"\",\"Platfrom\":\"\",\"QueryTime\":\"\",\"Account\":\"knavi\",\"ClientToken\":\"\",\"Language\":\"\",\"CNode\":{\"PCode\":\"CJFD\",\"SMode\":\"\",\"OperateT\":\"\"},\"QNode\":{\"SelectT\":\"\",\"Select_Fields\":\"\",\"S_DBCodes\":\"\",\"QGroup\":[{\"Key\":\"subject\",\"Logic\":1,\"Items\":[],\"ChildItems\":[{\"Key\":\"txt\",\"Logic\":1,\"Items\":[{\"Key\":\"txt_1\",\"Title\":\"\",\"Logic\":1,\"Name\":\"TI\",\"Operate\":\"%\",\"Value\":\"'<journal>'\",\"ExtendType\":0,\"ExtendValue\":\"\",\"Value2\":\"\"}],\"ChildItems\":[]}]}],\"OrderBy\":\"OTA|DESC\",\"GroupBy\":\"\",\"Additon\":\"\"}}";

    public static HtmlInfo html = new HtmlInfo();
    public static SimpleHttp http = new SimpleHttp();

    static String ua = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36";
    static String encode = "utf-8";
    //static String proxy = "localhost:8888";
    static String searchJournalUrl = "http://navi.cnki.net/knavi/Common/Search/Journal";
    static String searchEveryIssuePaperModelUrl = "http://navi.cnki.net/knavi/JournalDetail/GetArticleList?year=<year>&issue=<issue>&pykm=<PYKM>&pageIdx=0&pcode=CJFD";
    static String[] issueList = {"01","02","03","04","05","06","07","08","09","10","11","12"};
    //期刊名称
    static String journal = null;

    //查询所有md5，防止重复
    static String allMd5;

    public static void main(String[] args) throws InterruptedException {
        allMd5 = CnkiJournalPaperSaveDataToSql.getAllMd5();
        Map journalShortName = new HashMap();
        journalShortName.put("软件学报","RJXB");
        journalShortName.put("中国科学:信息科学","PZKX");
        journalShortName.put("自动化学报","MOTO");
        journalShortName.put("图书与情报","BOOK");
        journalShortName.put("图书情报工作","TSQB");
        journalShortName.put("电子政务","DZZW");
        journalShortName.put("机器人","JQRR");
        journalShortName.put("控制与决策","KZYC");
        journalShortName.put("遥感学报","YGXB");
        journalShortName.put("中国图象图形学报","ZGTB");
        journalShortName.put("控制理论与应用","KZLY");
        journalShortName.put("传感技术学报","CGJS");
        journalShortName.put("模式识别与人工智能","MSSB");
        journalShortName.put("情报科学","QBKX");
        journalShortName.put("情报学报","QBXB");
        journalShortName.put("计算机应用研究","JSYJ");
        journalShortName.put("计算机辅助设计与图形学学报","JSJF");
        journalShortName.put("数据分析与知识发现","XDTQ");
        journalShortName.put("计算机科学与探索","KXTS");
        journalShortName.put("计算机应用","JSJY");
        journalShortName.put("计算机科学","JSJA");
        journalShortName.put("中文信息学报","MESS");
        journalShortName.put("控制工程","JZDF");
        journalShortName.put("智能系统学报","ZNXT");
        journalShortName.put("信息与控制","XXYK");
        journalShortName.put("图学学报","GCTX");
        journalShortName.put("传感器与微系统","CGQJ");
        journalShortName.put("计算机学报","JSJX");

        CnkiJournalPaperConfig.init("config/cnkiJournalPaperConfig.txt");
        html.setUa(ua);
        html.setEncode(encode);

        if(CnkiJournalPaperConfig.entranceUrl != null && CnkiJournalPaperConfig.journalName != null){
            journal = CnkiJournalPaperConfig.journalName;
            String [] issues = getYearAndIssue(CnkiJournalPaperConfig.entranceUrl);
            int year = Integer.parseInt(issues[0]);
            int issue = Integer.parseInt(issues[1]);
            //查询期刊
            searchJournal();
            logger.info("15s后获取期刊详情");
            Thread.sleep(15 * 1000);
            //year<当前年份
            for(;year <= 2018;year++){
                for(;issue < 13;issue++){
                    String url = searchEveryIssuePaperModelUrl.replace("<year>",year + "").replace("<issue>",issueList[issue - 1])
                            .replace("<PYKM>",(String)journalShortName.get(journal));
                    queryList(url);
                }
                issue = 1;
            }
            //执行完的期刊存储下来
            FileOperation.appendWrite(journal + ";","config/cnkiJournalPaperExecutedJournalName.txt");
            journal = "";
        }

        while (true){
            //期刊名字列表
            List<String> journalNames = CnkiJournalPaperSaveDataToSql.getAllJournalNames();//FileOperation.read("config/cnkiJournalPaperJournalNames.txt").split("\n");

            //读取出已经执行完的关键词
            String executedJournalNames = FileOperation.read("config/cnkiJournalPaperExecutedJournalName.txt");
            if(executedJournalNames == null){
                executedJournalNames = "";
            }
           nextJournal: for(int i = 0;i < journalNames.size();i++){
                if("中国科学：信息科学".equals(journalNames.get(i))){
                    journalNames.set(i,"中国科学:信息科学");
                }
                if(executedJournalNames.contains(journalNames.get(i))){
                    continue;
                }
                journal = journalNames.get(i).trim();
                //查询期刊
                searchJournal();
                logger.info("15s后获取期刊详情");
                Thread.sleep(15 * 1000);
                int year = 2010;
                int issue = 1;
                if(journal.equals("中文信息学报")){
                    year = 2013;
                    issue = 4;
                }else if(journal.equals("控制理论与应用")){
                    year = 2010;
                    issue = 3;
                }else if(journal.equals("计算机应用研究")){
                    year = 2010;
                    issue = 10;
                }else if(journal.equals("图书与情报")){
                    year = 2018;
                    issue = 3;
                }else if(journal.equals("数据分析与知识发现")){
                    year = 2013;
                    issue = 2;
                }else if(journal.equals("图书情报工作")){
                    year = 2012;
                    issue = 3;
                }else if(journal.equals("智能系统学报")){
                    year = 2014;
                    issue = 4;
                }else if(journal.equals("模式识别与人工智能")){
                    year = 2014;
                    issue = 10;
                }
                for(;year <=2018;year++){
                    for(;issue < 13;issue++){
                        String url = searchEveryIssuePaperModelUrl.replace("<year>",year + "").replace("<issue>",issueList[issue - 1])
                                .replace("<PYKM>",(String)journalShortName.get(journal));
                        queryList(url);
                    }
                    issue = 1;
                }
                //执行完的期刊存储下来
                FileOperation.appendWrite(journal + ";","config/cnkiJournalPaperExecutedJournalName.txt");
            }
            //获取更新的数据
            for(int i = 0;i < journalNames.size();i++){
                Calendar calendar = Calendar.getInstance();
                for(int year = 2018;year <= calendar.get(Calendar.YEAR);year++){
                    for(int issue = 1;issue < 13;issue++){
                        String url = searchEveryIssuePaperModelUrl.replace("<year>",year + "").replace("<issue>",issueList[issue - 1])
                                .replace("<PYKM>",(String)journalShortName.get(journalNames.get(i).trim()));
                        queryList(url);
                    }
                }
            }
            //期刊全部查询完之后，等待一天
            logger.info("期刊全部采集完毕，等待24小时后继续采集,下次开始时间为："+ new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date(new Date().getTime() + 1000 * 60 * 60 * 24)));
            Thread.sleep(1000 * 60 * 60 * 24);
        }
    }
    //查询期刊
    public static void searchJournal(){
        ArrayList<String> params = new ArrayList<>();
        params.add("SearchStateJson##" + SearchStateJson.replace("<journal>",journal));
        params.add("displaymode##1");
        params.add("pageindex##1");
        params.add("pagecount##21");
        params.add("index##1");
        params.add("random##" + Math.random());
        html.setOrignUrl(searchJournalUrl);
        http.simplePost(html,params);
    }

    public static void queryList(String url) throws InterruptedException {

        logger.info("当前url为：" + url);
        FileOperation.appendWrite("当前url：" + url + "\n","config/cnkiJournalPaperUrl.txt");
        logger.info("当前期刊名称为：" + journal);
        FileOperation.appendWrite("当前期刊名称为：" + journal + "\n","config/cnkiJournalPaperUrl.txt");
        List<CnkiJournalPaperData> paperList = new ArrayList<>();
        //获取期刊的期数
        String[] issue = getYearAndIssue(url);

        html.setOrignUrl(url);
        http = new SimpleHttp();
        http.simplePost(html,new ArrayList<>());
        if(html.getContent() != null && html.getContent().indexOf("暂无目录信息") == -1){
            DocumentFragment paperNode = DomTree.getNode(html.getContent(),html.getEncode());
            NodeList paperNodeList = DomTree.commonList(CnkiJournalPaperXpath.paper_url,paperNode);
            if(paperNodeList != null){
                for(int i = 0;i < paperNodeList.getLength();i++){
                    try{
                        logger.info("15s后开始获取第" + (i + 1) + "篇文章的信息");
                        Thread.sleep(15 * 1000);
                        CnkiJournalPaperData paperData = new CnkiJournalPaperData();

                        String paperUrl = urlPrefix + "/knavi/" + paperNodeList.item(i).getTextContent();
                        html.setReferUrl("http://navi.cnki.net/knavi/JournalDetail?pcode=CJFD&pykm=JSJX");
                        html.setOrignUrl(paperUrl);
                        http.simpleGet(html);

                        DocumentFragment paperInfoNode = DomTree.getNode(html.getContent(),html.getEncode());
                        //获取文章题目
                        NodeList paperTitleNodeList = DomTree.commonList(CnkiJournalPaperXpath.TITLE,paperInfoNode);
                        if(paperTitleNodeList != null && paperTitleNodeList.getLength() > 0){
                            paperData.setTitle(paperTitleNodeList.item(0).getTextContent());
                        }
                        //获取文章摘要
                        NodeList paperBriefNodeList = DomTree.commonList(CnkiJournalPaperXpath.BRIEF,paperInfoNode);
                        if(paperBriefNodeList != null && paperBriefNodeList.getLength() > 0){
                            paperData.setBrief(paperBriefNodeList.item(0).getTextContent());
                        }
                        //判断MD5在数据库是否已经存在,存在则跳过，不存在则将md5保存到allMd5中
                        paperData.setMd5(MD5Util.MD5(paperData.getTitle() + paperData.getBrief()));
                        if(allMd5.contains(paperData.getMd5())){
                            logger.info("该文章已存在数据库中，跳过采集");
                            continue;
                        }
                        allMd5 = allMd5 + paperData.getMd5() + ",";
                        String authorUrlModel = "http://kns.cnki.net/kcms/detail/knetsearch.aspx?sfield=au&skey=<authorName>&code=<authorCode>";

                        //获取文章作者
                        NodeList paperAuthorNodeList = DomTree.commonList(CnkiJournalPaperXpath.AUTHOR,paperInfoNode);
                        if(paperAuthorNodeList != null){
                            String author = "";
                            String authorAffiliation = "";
                            for(int j = 0;j < paperAuthorNodeList.getLength();j++){
                                author = author + paperAuthorNodeList.item(j).getTextContent()+";";
                                //获取每一个作者对应的机构
                                String []authorNameAndCode;
                                logger.info("15s后获取第"+( j + 1 )+"个作者的机构信息");
                                Thread.sleep(15 * 1000);
                                try{
                                    authorNameAndCode = paperAuthorNodeList.item(j).getAttributes().getNamedItem("onclick").getTextContent().split("\\(")[1].split("\\)")[0].split(",");
                                    html.setOrignUrl(authorUrlModel.replace("<authorName>",authorNameAndCode[1].replace("'",""))
                                            .replace("<authorCode>",authorNameAndCode[2].replace("'","")));
                                    http.simpleGet(html);
                                    DocumentFragment authorInfoNode = DomTree.getNode(html.getContent(),html.getEncode());
                                    NodeList authorAffiliationNodeList = DomTree.commonList(CnkiJournalPaperXpath.authorAffiliation,authorInfoNode);
                                    if(authorAffiliationNodeList != null){
                                        authorAffiliation = authorAffiliation + authorNameAndCode[1].replace("'","") + "<:>";
                                        for(int a = 0;a < authorAffiliationNodeList.getLength();a++){
                                            authorAffiliation = authorAffiliation + authorAffiliationNodeList.item(a).getTextContent() + "<,>";
                                        }
                                        authorAffiliation = authorAffiliation + "<;>";
                                    }
                                }catch (Exception e){

                                }
                            }
                            paperData.setAuthorAffiliation(authorAffiliation);
                            paperData.setAuthor(author);
                        }
                        //获取文章机构
                        NodeList paperAffiliationNodeList = DomTree.commonList(CnkiJournalPaperXpath.affiliation,paperInfoNode);
                        if(paperAffiliationNodeList != null){
                            String aff = "";
                            for(int j = 0;j < paperAffiliationNodeList.getLength();j++){
                                aff = aff + paperAffiliationNodeList.item(j).getTextContent()+";";
                            }
                            paperData.setAffiliation(aff);
                        }
                        //获取文章下载量
                        NodeList paperDownloadNumNodeList = DomTree.commonList(CnkiJournalPaperXpath.PAPER_DOWNLOAD,paperInfoNode);
                        if(paperDownloadNumNodeList != null && paperDownloadNumNodeList.getLength() > 0){
                            paperData.setDownload_num(paperDownloadNumNodeList.item(0).getTextContent().trim());
                        }
                        //获取文章页码
                        NodeList paperPageNodeList = DomTree.commonList(CnkiJournalPaperXpath.pagination,paperInfoNode);
                        if(paperPageNodeList != null && paperPageNodeList.getLength() > 0){
                            paperData.setPage(paperPageNodeList.item(0).getTextContent());
                        }
                        //获取文章的基金、关键词、分类号
                        NodeList paperFundKeywordAndClassifyNodeList = DomTree.commonList(CnkiJournalPaperXpath.fundKeywordAndClassify,paperInfoNode);
                        if(paperFundKeywordAndClassifyNodeList != null){
                            for(int j = 0;j < paperFundKeywordAndClassifyNodeList.getLength();j++){
                                String content = paperFundKeywordAndClassifyNodeList.item(j).getTextContent();
                                if(content.contains("基金：")){
                                    paperData.setFund(content);
                                }else if(content.contains("关键词：")){
                                    paperData.setKeywords(content);
                                }else if(content.contains("分类号：")){
                                    paperData.setClass_num(content);
                                }
                            }
                        }
                        //获取期刊的期数
                        paperData.setIssue(issue[0] + "年" + issue[1] + "期");
                        //获取期刊名称
                        paperData.setJournal(journal);

                        logger.info("15s后获取文章的年度被引数");
                        Thread.sleep(15 * 1000);
                        //获取文章的被引量
                        String paperYearCiteUrl = "http://kns.cnki.net/kcms/detail/block/refyear.aspx?&vl=&" + html.getRealUrl().split("\\?")[1];
                        html.setOrignUrl(paperYearCiteUrl);
                        http.simpleGet(html);
                        paperData.setCite_num(manageYearCiteNum(Integer.parseInt(issue[0])));

                        paperList.add(paperData);
                    }catch (Exception e){
                        logger.info("获取第" + (i + 1) + "篇文章信息失败,继续获取下一篇文章");
                    }
                }
                logger.info("该页数据获取完成，存储到数据库");
                CnkiJournalPaperSaveDataToSql.saveCnkiJournalPaperToSql(paperList);
                logger.info("数据存储完成，size= " + paperList.size() + ",15s后继续获取数据");
                Thread.sleep(15 * 1000);
            }
        }

    }

    //获取期刊的期数
    public static String[] getYearAndIssue(String withYearAndIssueUrl){
        String year = withYearAndIssueUrl.split("year=")[1].split("&")[0];
        String issue = withYearAndIssueUrl.split("issue=")[1].split("&")[0];
        String [] result = {year,issue} ;
        return result;
    }
    //处理年度被引数
    public static String manageYearCiteNum(int year){
        String htmlContent = html.getContent();
        if(htmlContent == null || htmlContent.length() < 400){
            return "";
        }
        String allYearCiteNum = DomAnalysisXml(htmlContent.replace("text=\"中国期刊全文数据库\"","")).replace("null","");
        String []everyYear = allYearCiteNum.split(";");
        StringBuilder sb = new StringBuilder();
        for(int i = 0;i < everyYear.length;i++){
            String yearStr = everyYear[i].split(":")[0];
            int citeYear;
            if(yearStr.length() == 4){
                try{
                    citeYear = Integer.parseInt(yearStr);
                    if(citeYear > year - 1){
                        sb.append(everyYear[i] + ";");
                    }
                }catch (Exception e ){
                    e.printStackTrace();
                }
            }
        }
        return sb.toString();
    }

    //解析知网文章的年度被引数
    public static String DomAnalysisXml(String yearsCiteNumXml){
        //年度被引数
        String yearsCiteNum = null;
        //创建一个DocumentBuilderFactory的对象
        DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
        //创建DocumentBuilder对象
        DocumentBuilder db = null;
        try {
            InputStream is = new ByteArrayInputStream(yearsCiteNumXml.getBytes());
            db = dbf.newDocumentBuilder();
            //通过DocumentBuilder对象的parser方法加载XML字符串
            Document document = db.parse(is);
            //获取所有Item节点的集合
            NodeList itemList = document.getElementsByTagName("Item");
            //遍历每一个Item节点
            for (int i = 0; i < itemList.getLength(); i++) {
                //通过 item(i)方法 获取一个Item节点，nodelist的索引值从0开始
                Node item = itemList.item(i);
                //解析Item节点的子节点
                NodeList childNodes = item.getChildNodes();
                String year = null;
                String citing = null;
                String subCiting = null;
                //遍历childNodes获取每个节点的节点名和节点值
                for (int k = 0; k < childNodes.getLength(); k++) {
                    //区分出text类型的node以及element类型的node
                    if (childNodes.item(k).getNodeType() == Node.ELEMENT_NODE) {
                        if("YEAR".equals(childNodes.item(k).getNodeName())){
                            year = childNodes.item(k).getFirstChild().getNodeValue();
                        }else if("CITING".equals(childNodes.item(k).getNodeName())){
                            citing = childNodes.item(k).getFirstChild().getNodeValue();
                        }else if("SUB_CITING".equals(childNodes.item(k).getNodeName())){
                            subCiting = childNodes.item(k).getFirstChild().getNodeValue();
                        }
                    }
                }
                yearsCiteNum = yearsCiteNum + year + ":" + citing + "," + subCiting + ";";
            }
            return yearsCiteNum;
        } catch (ParserConfigurationException e) {
            e.printStackTrace();
        } catch (SAXException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        }
        return null;
    }
}
